﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Collections;
using System.IO;
using System.Diagnostics;
using log4net;
using System.Runtime.InteropServices;
using RollBackLib.src.utils;
using System.Text.RegularExpressions;
using RollbackLib.src.sys;

namespace RollBackLib.src.core {
    /// <summary>
    /// The current operation performed by the backup thread
    /// </summary>
    public enum CurrentOperation
    {
        Starting,
        Paused,             /// Going into the pause state
        InPause,            /// In pause mode
        Resumed,
        Finished,
        EarlyExit,
        StatusUpdate,
        SystemError,

        InvalidEntity,      /// the object was neither a file nor a directory
        AccessDenied,
        GeneralException,   /// An exception was thrown.
        CopyFileStarting,   /// the file copy operation is going to start
        CopyFileInProgress, /// the file copy operation is in progress
        CopyFileEnd,        /// the file copy process has ended

        CopyDirectoryBegin, /// Called when a directory copy is started
        CopyDirectoryEnd,   /// Called when directory copy ends without interruptions like stop or errors
        CopyDirectorySkipped, /// Directory copy was skipped because of filters
        CreatingDirectory,

        PruningStart,
        PruningEnd,
        PruningFileItem,        /// An file/directory is being deleted from the destination 
        PruningDirectoryItem,   /// 
        PruneException,         /// An exception during the pruning operation
                                
        ScheduledSave,          /// Call the updae handler to save all backup jobs.
    };

    /// <summary>
    /// Performs backup in the background using a thread.
    /// 
    /// </summary>
    class ThreadedBackup
    {
        #region member variables
        public static int PROGRESS_CONTINUE = 0;
        public static int PROGRESS_CANCEL = 1;
        public static int PROGRESS_STOP = 2;
        public static int PROGRESS_QUIET = 3;

        public static int CALLBACK_CHUNK_FINISHED = 0x00000000;
        public static int CALLBACK_STREAM_SWITCH = 0x00000001;
        //
        // Define CopyFileEx option flags
        //
        public static int COPY_FILE_FAIL_IF_EXISTS = 0x00000001;
        public static int COPY_FILE_RESTARTABLE = 0x00000002;
        public static int COPY_FILE_OPEN_SOURCE_FOR_WRITE = 0x00000004;
        public static int COPY_FILE_ALLOW_DECRYPTED_DESTINATION = 0x00000008;

        // windows error codes
        public const int FILE_NOT_FOUND = 2;

        public const int READ_PERMISSION_DENIED = 5;

        public const int WRITE_PERMISSION_DENIED = 32;

        private delegate Int32 CopyCallBackDelegate(long TotalFileSize, long BytesTransfered, long StreamSize, long StreamBytesTransfered, uint DwStreamNumber, long dwCallbackReason, long hSourceFile, long hDestinationFile, long lpData);

        private AutoResetEvent m_backupJobFinishEvent = new AutoResetEvent(false);

        [DllImport("kernel32.dll", EntryPoint = "CopyFileExW", SetLastError = true, CharSet=CharSet.Auto, PreserveSig=true)]
        private static extern Int32 CopyFileEx(
            string lpExistingFileName,
            string lpNewFileName,
            [MarshalAs(UnmanagedType.FunctionPtr)] CopyCallBackDelegate copycallback,
            long lpData,
            bool pbCancel,
            Int32 dwCopyFlags);

        
        [DllImport("kernel32.dll")]
        extern static int GetLastError();

        CopyCallBackDelegate m_copyCallback;
        //private Int64 m_copyExLParam = 0;

        private static readonly ILog log = LogManager.GetLogger(typeof(ThreadedBackup));

        private volatile bool m_running = false;
        private volatile bool m_paused = false;
        /// <summary>
        /// is true if the backup was stopped abruptly
        /// </summary>
        public volatile bool m_EarlyExit = false;

        private volatile AutoResetEvent m_pauseEvent = new AutoResetEvent(false);

        /// <summary>
        /// setting this to true just causes a dry run
        /// </summary>
        private bool m_testOnly = false;

        /// <summary>
        /// The System thread which calls the backup function.
        /// </summary>
        private Thread m_backupThread;

        /// <summary>
        /// All the data required to run a threded backup job.
        /// </summary>
        private BackupInfo m_bInfo;

        private byte[] m_tempData = new byte[1024 * 1024];
        /// <summary>
        /// Callback to be supplied to the dirWalk function.
        /// If this function returns false then dirWalk stops the directory walking.
        /// </summary>
        /// <param name="info"></param>
        /// <returns></returns>
        public delegate bool dirWalkCallback(FileSystemInfo info,int mode,ref bool skipRecursion);
        public const int DIRWALK_TOP_LEVEL = 1;
        public const int DIRWALK_SUB_LEVEL = 2;
        #endregion

        public ThreadedBackup() {
            m_copyCallback = new CopyCallBackDelegate(CopyProgressUpdate);
        }

        #region Utility functions
        /// <summary>
        /// returns true if the source file is newer than the target
        /// </summary>
        /// <param name="src"></param>
        /// <param name="dest"></param>
        /// <returns></returns>
        private bool isSourceFileNewer(FileInfo src, FileInfo dest)
        {
            bool newer =
            (src.LastWriteTimeUtc > dest.LastWriteTimeUtc)
            || (src.Length != dest.Length);
            return newer;
        }


        private bool shouldInclude(string fullName,
            List<Regex> includeFilter, 
            List<Regex> excludeFilter,
            ref string exclusionReason)
        {
            bool include = true;

            if (includeFilter.Count != 0)
            {
                // if include filter is non-empty then 'fullName' must satisfy the include filter
                include = false;
                exclusionReason = "Not in include filter";  
            }

            foreach (Regex r in excludeFilter)
            {
                MatchCollection m = r.Matches(fullName);
                if (m.Count > 0)
                {
                    include = false;
                    exclusionReason = r.ToString();
                    break;
                }
            }

            foreach(Regex r in includeFilter) 
            {
                MatchCollection m = r.Matches(fullName);
                if (m.Count > 0)
                {
                    include = true;
                    break;
                }
            }
            return include;
        }

        /// <summary>
        /// Remove the read-only attribute of the file
        /// </summary>
        /// <param name="destFileName"></param>
        private void fixFileAttributes(string destFileName)
        {
            if (File.Exists(destFileName))
            {
                FileAttributes attr = File.GetAttributes(destFileName);
                attr = (attr & ~FileAttributes.ReadOnly);
                attr = (attr & ~FileAttributes.Hidden);
                attr = (attr & ~FileAttributes.System);
                File.SetAttributes(destFileName, attr);
            }
        }

        private void CreateDir(string srcDir, string destDir)
        {
            if (!Directory.Exists(destDir))
            {
                notifyDirectoryOperation(CurrentOperation.CreatingDirectory, srcDir , destDir);
                if (!m_testOnly)
                {
                    Directory.CreateDirectory(destDir);
                }
            }
        }

        private void CreateDir(string dest)
        {
            CreateDir(null, dest);
        }

        private void DeleteFile(string file)
        {
            m_bInfo.m_Statistics.m_currentDestination = file;
            notifyOperation(CurrentOperation.PruningFileItem, "Deleting file <" + file + ">");
            if (!m_testOnly)
            {
                FileInfo finfo = new FileInfo(file);
                FileAttributes attr = FileAttributes.ReadOnly | FileAttributes.Hidden | FileAttributes.System;
                FileAttributes newAttr = finfo.Attributes & (~attr);
                finfo.Attributes = newAttr;
                finfo.Delete();
            }
        }

        private void DeleteDir(string dir, bool recursive)
        {
            m_bInfo.m_Statistics.m_currentDestination = dir;
            notifyOperation(CurrentOperation.PruningDirectoryItem, "Deleting Directory <" + dir + ">. Recursive="+recursive);
            if (!m_testOnly)
            {
                Directory.Delete(dir, recursive);
            }
        }
        #endregion

        #region NotifyOperation functions
        public static void notifyOperation(BackupInfo bInfo, CurrentOperation opr, FileInfo file,string msg, long fileSize,bool skipping)
        {
            bInfo.m_Statistics.m_operation = opr;
            bInfo.m_Statistics.m_operationMsg = msg;
            bool callUpdate = true;

            switch (opr)
            {
                case CurrentOperation.EarlyExit:
                    // we notify the update handler about the earlyexit only once per backup job invocation
                    if (bInfo.m_Statistics.m_earlyExitNotified)
                    {
                        callUpdate = false;
                    }
                    else
                    {
                        bInfo.m_Statistics.m_earlyExitNotified = true;
                    }
                    break;
                case CurrentOperation.CopyFileEnd:
                    if (!skipping)
                    {
                        ++bInfo.m_Statistics.m_fileStats.m_copied;
                        bInfo.m_Statistics.m_fileStats.m_totalBytesCopied += fileSize;
                    }
                    else
                    {
                        ++bInfo.m_Statistics.m_fileStats.m_skipped;
                    }
                    bInfo.m_Statistics.m_fileStats.m_totalBytesScanned += fileSize;
                    break;

                case CurrentOperation.PruningFileItem:
                    ++bInfo.m_Statistics.m_fileStats.m_deleted;
                    break;
                case CurrentOperation.PruningDirectoryItem:
                    bInfo.m_Statistics.m_dirStats.m_deleted++;
                    break;
            }

            bInfo.m_Statistics.collectStatistics(file);
            if (callUpdate)
            {
                // call the update handler
                bInfo.callUpdateHandler( );
            }
        }

        private void notifySystemError(FileInfo src,string msg,Exception e)
        {
            int lastError = GetLastError();
            msg = msg +
                "\r\n" +
                "WinError=" + lastError;
            m_bInfo.m_Statistics.m_fileStats.m_numErrors++;
            notifyOperation(m_bInfo,CurrentOperation.SystemError, src, msg, src.Length, true /*skipping*/);
        }


        private void notifyException(CurrentOperation currentOperation, string msg, Exception e)
        {
            int lastError = GetLastError();
            msg = msg +
                "\r\n" +
                e.Message +
                "\r\n" +
                "WinError=" + lastError;
            this.m_bInfo.m_LastException = e;
            this.m_bInfo.m_LastExceptionDisplayed = false;
            m_bInfo.m_Statistics.m_fileStats.m_numErrors++;
            notifyOperation(m_bInfo, currentOperation, null /* file info */, msg, 0 /* file size */, false /* skipping */);
            this.m_bInfo.m_LastExceptionDisplayed = true;
        }

        private void notifyOperation(CurrentOperation currentOperation, string msg)
        {
            notifyOperation(m_bInfo, currentOperation, null, msg, 0, true /*skipping*/);
        }

        private void notifyDirectoryOperation(CurrentOperation currentOperation, string srcDir, string destDir)
        {
            notifyDirectoryOperation(currentOperation, srcDir, destDir, "");
        }

        private void notifyDirectoryOperation(CurrentOperation currentOperation, string srcDir, string destDir,string additionalMsg)
        {
            string msg = "";
            bool skipping = false;
            
            m_bInfo.m_Statistics.m_currentSource = srcDir;
            m_bInfo.m_Statistics.m_currentDestination = destDir;

            switch (currentOperation)
            {
                case CurrentOperation.CopyDirectoryBegin:
                    msg = "Copying directory <" + srcDir + "> to <" + destDir + ">";
                    break;
                case CurrentOperation.CopyDirectoryEnd:
                    msg = "Finished copying directory <" + srcDir + ">";
                    ++m_bInfo.m_Statistics.m_dirStats.m_copied;
                    break;
                case CurrentOperation.CreatingDirectory:
                    m_bInfo.m_Statistics.m_dirStats.m_created++;
                    msg = "Creating directory <" + destDir + ">";
                    break;
                case CurrentOperation.EarlyExit:
                    msg = "User Aborted while copying Directory <" + srcDir + ">";
                    break;
                case CurrentOperation.CopyDirectorySkipped:
                    ++m_bInfo.m_Statistics.m_dirStats.m_skipped;
                    skipping = true;
                    msg = "Skipping directory <" + srcDir + ">. Excluded by folder filter <" + additionalMsg + ">";
                    break;
                default:
                    msg = "Invalind operation specified <" + currentOperation + ">. SrcDir=<" + srcDir + "> DestDir=<" + destDir + ">";
                    Debug.Assert(false, msg);
                    break;
            }
            notifyOperation(m_bInfo, currentOperation, null /* file info */, msg, 0 /* file size */, skipping);
        }
        #endregion

        #region Copy File and Directory functions
        /// <summary>
        /// Callback supplied to CopyFileEx.
        /// </summary>
        /// <param name="TotalFileSize"></param>
        /// <param name="BytesTransfered"></param>
        /// <param name="StreamSize"></param>
        /// <param name="StreamBytesTransfered"></param>
        /// <param name="DwStreamNumber"></param>
        /// <param name="dwCallbackReason"></param>
        /// <param name="hSourceFile"></param>
        /// <param name="hDestinationFile"></param>
        /// <param name="lpData"></param>
        /// <returns></returns>
        private Int32 CopyProgressUpdate(long TotalFileSize,
            long BytesTransfered,
            long StreamSize,
            long StreamBytesTransfered,
            uint DwStreamNumber,
            long dwCallbackReason,
            long hSourceFile,
            long hDestinationFile,
            long lpData)
        {
            // This takes the current stream size (amount transfered) and creates a percentage based on file size
            Int32 ret = 0;
            if (ShouldExit())
            {
                ret = 1;
            }
            else
            {
                // save the number of bytes copied so far
                m_bInfo.m_Statistics.m_fileStats.m_inProgress = BytesTransfered;
                int percent = TotalFileSize == 0 ? 0 : (int)((BytesTransfered * 100L) / TotalFileSize);
                string msg =
                    "Copying <"
                    + m_bInfo.m_Statistics.m_currentSource
                    + "> to <"
                    + m_bInfo.m_Statistics.m_currentDestination
                    + "> "
                    + AppUtils.toHumanReadableSize(BytesTransfered)
                    + " of "
                    + AppUtils.toHumanReadableSize(TotalFileSize)
                    + " (" + percent + "%)";
                ;

                notifyOperation(m_bInfo, CurrentOperation.CopyFileInProgress, null, msg /* pass the existing message back*/ , BytesTransfered, false /* skipping */);
            }
            return (Int32)ret;
        }

        /// <summary>
        /// Copy directory (including empty directories) and its subdirectories to the destDir location
        /// </summary>
        /// <param name="src"></param>
        /// <param name="destDir"></param>
        private void copyDirectory(DirectoryInfo srcDirInfo, string fullDestDirName) {
            string fullSrcDirName = "";
            try
            {
                fullSrcDirName = srcDirInfo.Name; // first attempt to get the name so we have something in case of an exception while trying to access the fullname.
                fullSrcDirName = srcDirInfo.FullName;
            }
            catch (Exception )
            {
                // ignore the exception generated during trying to get the name/fullname of the directory.
            }

            try
            {
                do
                {
                    string exclusionReason = "";
                    bool include = false;

                    Dictionary<string, string> pruneList = new Dictionary<string, string>(); // contents of the destination directory
                    long dirProgressCount = 0;
                    bool isDelete = m_bInfo.isDeleteOperation();
                    // if this is the delete operation then just add the files/directories to the prunelist.
                    bool shouldPrune = isDelete || m_bInfo.m_Job.m_SynchronizeBackups;
                    if (isDelete)
                    {
                        include = true;
                    }
                    else
                    {
                        include = shouldInclude(fullSrcDirName, m_bInfo.m_includeFolderFilter, m_bInfo.m_excludeFolderFilter, ref exclusionReason);
                    }

                    if (include)
                    {
                        if (!Directory.Exists(fullDestDirName))
                        {
                            if (isDelete)
                            {
                                // the destination directory does not exist.
                                break;
                            }
                            else
                            {
                                // the destination directory does not exist and this is not a delete operation.
                                CreateDir(fullSrcDirName,fullDestDirName);
                                shouldPrune = false;    // if the directory did not exist then there is no need to add it to the prune list
                            }
                        }

                        DirectoryInfo destDirInfo = new DirectoryInfo(fullDestDirName);
                        if (shouldPrune)
                        {
                            FileSystemInfo[] allItems = destDirInfo.GetFileSystemInfos();
                            // the destination directory already exists, get the contents of it.
                            // put all the items into a map for easy search, there is no need to worry about
                            // duplicates since the file/dir name has to be unique within a directory
                            foreach (FileSystemInfo info in allItems)
                            {
                                string name = fullDestDirName + "\\" + info.Name;
                                addToPruneList(pruneList, name);
                            }
                        }

                        m_bInfo.m_Statistics.m_dirStats.m_inProgress = 0;
                        DirectoryInfo[] dinfos = null;
                        FileInfo[] finfos = null;
                        long totalCount = 0;

                        if (isDelete)
                        {
                            dinfos = destDirInfo.GetDirectories();
                            finfos = destDirInfo.GetFiles();
                            notifyOperation(CurrentOperation.StatusUpdate, "Scanning directory " + fullDestDirName);
                            totalCount = m_bInfo.m_Statistics.m_dirStats.m_currentSize = dinfos.Length + finfos.Length;
                        }
                        else
                        {
                            // copy the directory
                            dinfos = srcDirInfo.GetDirectories();
                            finfos = srcDirInfo.GetFiles();
                            notifyDirectoryOperation(CurrentOperation.CopyDirectoryBegin, fullSrcDirName, fullDestDirName);
                            totalCount = m_bInfo.m_Statistics.m_dirStats.m_currentSize = dinfos.Length + finfos.Length;
                            for (int i = 0; i < dinfos.Length; i++)
                            {
                                ++dirProgressCount;
                                string dest = fullDestDirName + "\\" + dinfos[i].Name;
                                // recursive call
                                copyDirectory(dinfos[i], dest);
                                // remove the directory entry from the map if the source location has that name
                                removeFromPruneList(pruneList, dest);
                                if (ShouldExit())
                                {
                                    break;
                                }
                            }

                            // copy all the files in the current directory
                            if (finfos.Length > 0)
                            {
                                notifyOperation(CurrentOperation.StatusUpdate, "Found " + finfos.Length + " items in <" + fullSrcDirName + ">");
                                for (int i = 0; i < finfos.Length; i++)
                                {
                                    m_bInfo.m_Statistics.m_dirStats.m_currentSize = totalCount;
                                    m_bInfo.m_Statistics.m_dirStats.m_inProgress = ++dirProgressCount;
                                    FileInfo fileInfo = finfos[i];
                                    string fullName = fullDestDirName + "\\" + fileInfo.Name;
                                    // remove the file entry from the map if the source location has that name
                                    removeFromPruneList(pruneList, fullName);
                                    copyFile(fileInfo, fullDestDirName);
                                    if (ShouldExit())
                                    {
                                        break;
                                    }
                                }
                                m_bInfo.m_Statistics.m_dirStats.m_inProgress = dirProgressCount;
                            }
                            notifyDirectoryOperation(CurrentOperation.CopyDirectoryEnd, fullSrcDirName, fullDestDirName);
                        }

                        // notify if the copy operation must exit because of user action.
                        // now delete the files and directories that are part of the destination but
                        // can no longer be found in the source.  This is the synchronization operation.
                        pruneFilesAndFolders(pruneList);
                    }
                    else
                    {
                        notifyDirectoryOperation(CurrentOperation.CopyDirectorySkipped, fullSrcDirName, fullDestDirName, exclusionReason);
                    }
                } while (false);

                if (ShouldExit())
                {
                    notifyDirectoryOperation(CurrentOperation.EarlyExit, fullSrcDirName, fullDestDirName);
                }
            }
            catch (AccessViolationException e)
            {
                string msg = "Got an access violation when copying <" + fullSrcDirName + ">";
                log.Warn(msg);
                notifyException(CurrentOperation.AccessDenied, msg, e);
            }
            catch (UnauthorizedAccessException e)
            {
                string msg = "Got an UnauthorizedAccessException when copying <" + fullSrcDirName + ">";
                log.Warn(msg);
                notifyException(CurrentOperation.AccessDenied, msg, e);
            }
            catch (IOException e)
            {
                string msg = "Caught an IOException when copying <" + fullSrcDirName + ">";
                log.Warn(msg, e);
                notifyException(CurrentOperation.AccessDenied, msg, e);
            }
            catch (Exception e)
            {
                string msg = "Caught an exception when copying <" + fullSrcDirName + ">";
                log.Warn(msg, e);
                notifyException(CurrentOperation.GeneralException, msg, e);
            }

        }

        private void removeFromPruneList(Dictionary<string, string> pruneList, string p)
        {
            bool prune = m_bInfo.m_Job.m_SynchronizeBackups || m_bInfo.isDeleteOperation();
            if (prune)
            {
                string lowerP = p.ToLowerInvariant();
                pruneList.Remove(lowerP);
            }
        }

        private void addToPruneList(Dictionary<string, string> pruneList, string p)
        {
            bool prune = m_bInfo.m_Job.m_SynchronizeBackups || m_bInfo.isDeleteOperation();
            if (prune)
            {
                string lowerStr = p.ToLowerInvariant();
                pruneList[lowerStr] = p;
            }
        }

        private bool pruneCallback2(FileSystemInfo info)
        {
            try
            {
                bool isDir = (info.Attributes & FileAttributes.Directory) != 0;
                if (isDir)
                {
                    DeleteDir(info.FullName, false);
                }
                else
                {
                    DeleteFile(info.FullName);
                }
            }
            catch (Exception e)
            {
                notifyException(CurrentOperation.PruneException, "Caught an exception while pruning", e);
            }
            return ShouldExit();
        }

        /// <summary>
        /// Prune the files/directories by looking at the supplied list.
        /// </summary>
        /// <param name="pruneList">List of files and directories to be pruned. Key is the case insensitive name and value is the name in which case is preserved</param>
        /// <param name="cbMode">Mode of operation DIR. One of DIRWALK_SUB_LEVEL, DIRWALK_TOP_LEVEL
        /// In the DIRWALK_SUB_LEVEL mode files/folders will be deleted unconditionally.
        /// In the DIRWALK_TOP_LEVEL mode files/folders that are not in the list of items to be backed up, will be deleted.
        /// </param>
        private void pruneFilesAndFolders(Dictionary<string, string> pruneList)
        {
            bool prune = m_bInfo.m_Job.m_SynchronizeBackups || m_bInfo.isDeleteOperation();
            if (prune)
            {
                if (pruneList.Count > 0)
                {
                    notifyOperation(CurrentOperation.PruningStart, "Starting pruning of <" + pruneList.Count + "> ");
                    DirWalker dirWalker = new DirWalker(pruneCallback2);
                    foreach (string s in pruneList.Keys)
                    {
                        string fullName = pruneList[s];
                        try
                        {
                            if (ShouldExit())
                            {
                                break;
                            }
                            if (dirWalker.walkDir(s))
                            {
                                break;
                            }
                        }
                        catch (Exception e)
                        {
                            notifyException(CurrentOperation.PruneException, "Error deleting <" + fullName + ">", e);
                        }
                    }
                    notifyOperation(CurrentOperation.PruningStart, "Finished pruning <" + pruneList.Count + "> items.");
                    pruneList.Clear();
                }
            }
        }

        private bool fileExistsInTopList(string testFile, ref bool exactMatch)
        {
            bool exists = false;
            string testFile2 = testFile.ToLowerInvariant();
            exactMatch = false;
            foreach (BackupItem backupItem in m_bInfo.m_Items)
            {
                string itemFile2 = backupItem.m_path.ToLowerInvariant();
                if (itemFile2.StartsWith(testFile2))
                {
                    exists = true;
                    if (itemFile2.Equals(testFile2))
                    {
                        exactMatch = true;
                        break;
                    }
                }
            }
            return exists;
        }

        private bool copyFile(FileInfo src, string parentDir)
        {
            string fullDestFileName = parentDir + "\\" + src.Name;
            bool copied = false;
            string fullSrcFileName = src.FullName;
            try
            {
                FileInfo dest = new FileInfo(fullDestFileName);
                string exclusionReason = "";
                bool include = shouldInclude(fullSrcFileName, m_bInfo.m_includeFileFilter, m_bInfo.m_excludeFileFilter, ref exclusionReason);
                bool newerFile = false;
                bool overwrite = false;

                if (include)
                {
                    if (File.Exists(fullDestFileName))
                    {
                        newerFile = isSourceFileNewer(src, dest);
                        if (!newerFile)
                        {
                            overwrite = m_bInfo.m_Job.m_OverwriteNewerFiles;
                        }                        
                    }
                    else
                    {
                        newerFile = true;
                    }                    
                }

                if ((newerFile || overwrite) && include)
                {
                    fixFileAttributes(fullDestFileName);
                    m_bInfo.m_Statistics.m_fileStats.m_currentSize = src.Length;

                    // set the source/dest paths
                    m_bInfo.m_Statistics.m_currentSource = src.FullName;
                    m_bInfo.m_Statistics.m_currentDestination = fullDestFileName;
                    CreateDir(parentDir);
                    if (src.LastWriteTime > DateTime.Now)
                    {
                        // check if the time is in future and warn about this
                        //notifyOperation(CurrentOperation.StatusUpdate, "File <" + fullDestFileName + "> has a timestamp in the future <" + src.LastWriteTime.ToLongDateString() + " " + src.LastWriteTime.ToLongTimeString() + "> ");
                    }
                    notifyOperation(m_bInfo, CurrentOperation.CopyFileStarting, src, "Copying File <" + fullSrcFileName + "> to " + parentDir, 0/*src.Length*/, false);

                    if (!m_testOnly)
                    {
                        // reset the number of bytes copied so far.
                        m_bInfo.m_Statistics.m_fileStats.m_inProgress = 0;

                        //Int32 ret = CopyFileEx(src.FullName, fullDestFileName, m_copyCallback, m_copyExLParam, cancel, COPY_FILE_RESTARTABLE);
                        long totalTransferred = 0;
                        using(BinaryReader infile = new BinaryReader(File.OpenRead(src.FullName)))
                        //using (BinaryReader infile = new BinaryReader(new FileStream(src.FullName, System.IO.FileMode.Open))) 
                        {
                            using (BinaryWriter outfile = new BinaryWriter(new FileStream(fullDestFileName,System.IO.FileMode.Create)))
                            {
                                int len = 0;
                                while( (len = infile.Read(m_tempData, 0, m_tempData.Length)) > 0) {
                                    totalTransferred += len;
                                    outfile.Write(m_tempData, 0, len);
                                    int ret = m_copyCallback(src.Length, totalTransferred, len, totalTransferred, 0, 0, 0, 0, 0);
                                    if (ret == 1)
                                    {
                                        break; // early exit
                                    }
                                }
                            };
                        }
                        m_bInfo.m_Statistics.m_systemError = GetLastError();
                        m_bInfo.m_Statistics.m_fileStats.m_inProgress = 0;
                    }

                    if (m_bInfo.m_Statistics.m_systemError == 1235 && m_bInfo.m_Statistics.m_backupWasStopped)
                    {
                        // ignore the error because backup was forcibly stopped
                        m_bInfo.m_Statistics.m_systemError = 0;
                    }

                    if (m_bInfo.m_Statistics.m_systemError != 0)
                    {
                        switch (m_bInfo.m_Statistics.m_systemError)
                        {
                            case FILE_NOT_FOUND:
                                notifySystemError(src, "File <" + fullSrcFileName + "> not found, while copying to " + parentDir,null);
                                break;
                            case READ_PERMISSION_DENIED:
                                notifySystemError(src, "Permission denied when copying file <" + fullSrcFileName + "> to " + parentDir,null);
                                break;
                            case WRITE_PERMISSION_DENIED:
                                notifySystemError(src, "Permission denied when copying file <" + fullSrcFileName + "> to " + parentDir,null);
                                break;
                            default:
                                notifySystemError(src, "Got system error=" + m_bInfo.m_Statistics.m_systemError + " while copying <" + fullSrcFileName + "> to " + parentDir,null);
                                break;
                        }
                        // reset the system error after it was reported
                        m_bInfo.m_Statistics.m_systemError = 0;
                        copied = false;
                    }
                    else if (m_running)
                    {
                        notifyOperation(m_bInfo, CurrentOperation.CopyFileEnd, src, "Copied <" + fullSrcFileName + "> to " + parentDir, src.Length, false);
                        copied = true;
                    }
                    else
                    {
                        // the copy operation was aborted by the user by clicking on stop
                        copied = false;
                    }
                }
                else
                {
                    string msg = (include == false) ? "Excluded by file filter <" + exclusionReason + ">" : "since it is up to date";
                    notifyOperation(m_bInfo, CurrentOperation.CopyFileEnd, src, "Skipping File <" + fullSrcFileName + ">. " + msg, src.Length, true);
                }
            }
            catch (Exception e)
            {
                string msg = "Got an Exception when copying <" + fullSrcFileName + "> to <" + fullDestFileName + ">";
                notifyException(CurrentOperation.GeneralException, msg, e);
            }
            return copied;
        }

        #endregion

        /// <summary>
        /// The main thread which runs the backup job
        /// </summary>
        /// <param name="threadedBackupInfoObject"></param>
        public void theRealBackupThread(object threadedBackupInfoObject ) {
            BackupInfo binfo = (BackupInfo)(threadedBackupInfoObject);
            binfo.m_Statistics.m_started = DateTime.Now;
            binfo.m_Statistics.m_BackupType = binfo.m_Statistics.m_BackupType;
            try
            {
                PowerState.PreventSystemPowerdown();
                m_pauseEvent.Reset();
                m_running = true;
                m_EarlyExit = true;
                // log messages when the job starts
                logJobStartMessage(binfo);
                // create the destination directory
                CreateDir(binfo.m_DestDir);

                Dictionary<string, string> pruneList = new Dictionary<string, string>();
                prepareTopLevelPruneList(pruneList);

                string src = "";
                string fullDestPath = "";
                //notifyException(CurrentOperation.AccessDenied, "Test exception", new Exception("Hello World"));

                foreach (BackupItem backupItem in binfo.m_Items)
                {
                    try
                    {
                        src = backupItem.m_path;
                        // convert the path to the backed up path
                        if (m_bInfo.isBackup() || m_bInfo.isDeleteOperation())
                        {
                            fullDestPath = convertDestToBackupPath(m_bInfo.m_DestDir, backupItem.m_path);
                        }
                        else
                        {
                            fullDestPath = convertDestToRestorePath(m_bInfo.m_DestDir, backupItem.m_path);
                        }

                        if (backupItem.m_enabled)
                        {
                            //notifyOperation(CurrentOperation.StatusUpdate, "Top level item <" + backupItem.m_path + ">");
                            if (Directory.Exists(src))
                            {
                                DirectoryInfo srcDInfo = new DirectoryInfo(src);
                                copyDirectory(srcDInfo, fullDestPath);
                            }
                            else if (File.Exists(src))
                            {
                                FileInfo finfo = new FileInfo(src);
                                string destDir = Directory.GetParent(fullDestPath).FullName;
                                copyFile(finfo, destDir);
                            }
                            else
                            {
                                notifyOperation(CurrentOperation.InvalidEntity, src + " does not exist or is a special file.");
                            }
                        }
                        else
                        {
                            notifyOperation(CurrentOperation.StatusUpdate, "Skipping top level item <" + backupItem.m_path + "> since it is disabled.");
                        }

                        if (ShouldExit())
                        {
                            notifyDirectoryOperation(CurrentOperation.EarlyExit, backupItem.m_path, "");
                            break;
                        }
                    }
                    catch (AccessViolationException e)
                    {
                        string msg = "Got an access violation when copying <" + src + ">";
                        log.Warn(msg);
                        notifyException(CurrentOperation.AccessDenied, msg,e);
                    }
                    catch (IOException e)
                    {
                        string msg = "Caught an IOException when copying <" + src + ">";
                        log.Warn(msg, e);
                        notifyException(CurrentOperation.AccessDenied, msg,e);
                    }
                    catch (Exception e)
                    {
                        string msg = "Caught an exception when copying <" + src + ">";
                        log.Warn(msg, e);
                        notifyException(CurrentOperation.GeneralException, msg, e);
                    }
                }
                if (!ShouldExit())
                {
                    m_EarlyExit = false;
                    if (binfo.isDeleteOperation())
                    {
                        addToPruneList(pruneList, binfo.m_DestDir);
                    }
                    pruneFilesAndFolders(pruneList);
                }
            }
            catch (Exception e)
            {
                m_bInfo.m_LastException = e;
                notifyException(CurrentOperation.GeneralException, "Got an exception ", e);
            }
            finally {
                binfo.m_Statistics.m_stopped = DateTime.Now;
                string line = binfo.createJobEndStatus( );
                PowerState.AllowSystemPowerdown();
                try
                {
                    notifyOperation(CurrentOperation.Finished, line);
                }
                finally
                {
                    m_running = false;
                    m_bInfo = null;
                    m_backupThread = null;
                    m_backupJobFinishEvent.Set();
                }
            }
        }

        /// <summary>
        /// Look at the destination directory and add those top level directories under 
        /// "backupDir"\local and "backupDir"\remote which no longer exist in the list of items to be backed up.
        /// e.g. if C:\Music and C:\Songs  used to exist in the top list and the user decided to remove
        /// C:\Songs from the top list then the syncrhonize operation must get rid of the backed up C:\Songs
        /// </summary>
        /// <param name="pruneList"></param>
        private void prepareTopLevelPruneList(Dictionary<string, string> pruneList)
        {
            bool isPrune = m_bInfo.isDeleteOperation() || (m_bInfo.isBackup() && m_bInfo.m_Job.m_SynchronizeBackups);
            // populate the prune list with the contents of the destination directory.
            if (isPrune)
            {
                // run through the files and directories in the destination directory and 
                // check to see if they exist in the source list.
                string finalDestDir = m_bInfo.m_Job.getFinalDestinationDir() + "\\local";
                DirectoryInfo dinfo = new DirectoryInfo(finalDestDir);
                FileSystemInfo[] levelOne = dinfo.GetFileSystemInfos();
                Dictionary<string, string> map = new Dictionary<string, string>();

                // add all the files/directory in the destination directory to a map
                if (levelOne != null)
                {
                    foreach (FileSystemInfo f in levelOne)
                    {
                        // go through C_ D_ etc
                        FileSystemInfo[] levelTwo = DirWalker.getFileSystemInfos(f.FullName);
                        if (levelTwo != null)
                        {
                            foreach (FileSystemInfo f2 in levelTwo)
                            {
                                if (ShouldExit())
                                {
                                    return;
                                }
                                string str = convertDestToRestorePath(m_bInfo.m_DestDir, f2.FullName);
                                map.Add(str.ToLowerInvariant(), str);
                            }
                        }
                    }
                }

                // remove the items in the backup list from the map
                bool exactMatch = false;
                Dictionary<string, string> removeList = new Dictionary<string, string>();
                foreach (string str in map.Keys)
                {
                    if (!fileExistsInTopList(str, ref exactMatch))
                    {
                        removeList.Add(str, str);
                    }
                    if (ShouldExit())
                    {
                        return;
                    }
                }
                // add the remaining items to the prunelist
                foreach (string str in removeList.Keys)
                {
                    if (ShouldExit())
                    {
                        return;
                    }
                    string path = convertDestToBackupPath(m_bInfo.m_DestDir,str);
                    addToPruneList(pruneList, path);
                }
            }
        }

        /// <summary>
        /// log some information when the backup starts
        /// </summary>
        /// <param name="binfo"></param>
        private void logJobStartMessage(BackupInfo binfo)
        {
            string startMsg =
                "Starting "
                + (binfo.m_Statistics.m_BackupType.ToString())
                + " at " + binfo.m_Statistics.m_started.ToLocalTime().ToString()
                + ".  Destination "
                + "<" + binfo.m_DestDir + ">"
                + " for "
                + binfo.m_Items.Count
                + " items.";

            string filterMsg =
                string.Format("(File include: {0}) (File exclude: {1}) (Folder include: {2}) (Folder exclude: {3})",
                m_bInfo.m_includeFileFilterStr,
                m_bInfo.m_excludeFileFilterStr,
                m_bInfo.m_includeFolderFilterStr,
                m_bInfo.m_excludeFolderFilterStr);

            string syncMessage = "Synchronization is " +
                (binfo.m_Job.m_SynchronizeBackups ? " enabled " : " not enabled ") +
                " for this job.";

            string notifyMsg = startMsg + " Filters are " + filterMsg + ".  " + syncMessage;
            notifyOperation(CurrentOperation.Starting, notifyMsg);
        }

        /// <summary>
        /// Convert the source path into a destination path by converting drive letters and unc
        /// prefix to path names
        /// </summary>
        /// <param name="src"></param>
        /// <returns></returns>
        public static string convertDestToBackupPath(string jobDestDir,string path)
        {
            string dir = System.IO.Path.GetFullPath(path);
            string prefix = System.IO.Path.GetPathRoot(dir);
            if (prefix[0] != '\\' && prefix[0] != '/')
            {
                // this is a non-unc path
                dir = dir.Replace(prefix, "");
                prefix = "local\\" + prefix.Replace(':', '_');
            }
            else
            {
                // this is a UNC path name
                prefix = "remote";
            }
            string fullDestPath = jobDestDir + "\\" + prefix + "\\" + dir;
            fullDestPath = Path.GetFullPath(fullDestPath);
            return fullDestPath;
        }

        /// <summary>
        /// Convert the path to the restore path.
        /// e.g. convert C_ to C:
        /// </summary>
        /// <param name="src"></param>
        /// <returns></returns>
        public static string convertDestToRestorePath(string jobDestDir,string src)
        {
            string dir = System.IO.Path.GetFullPath(src);
            string destDir = System.IO.Path.GetFullPath(jobDestDir);
            dir = dir.Replace(destDir, "");
            string local = "\\local\\";
            string remote = "\\remote\\";

            if(dir.StartsWith(local) )
            {
                // this is a local restore path
                dir = dir.Substring(local.Length);
                if (dir.Length > 1 && dir[1] == '_')
                {

                    dir = dir.Remove(1, 1); // remove the '_'
                    dir = dir.Insert(1, ":"); // replace it with ":"
                }
            }
            else if (dir.StartsWith(remote))
            {
                // this is a remote restore path
                dir = dir.Substring(remote.Length);
            }
            return dir;
        }

        /// <summary>
        /// Currently performs two functions of pausing if a puase was requested and after that
        /// checks if "Stop" was requested.
        /// </summary>
        /// <returns>True if stop was requested else returns false.</returns>
        public bool ShouldExit( ) {
            if (IsPaused())
            {
                waitInPausedState();
            }

            // the following code is for easy debugging, the compiler should optimize it out.
            bool ret = !m_running;
            if (ret)
            {
                return ret;
            }
            return ret;
        }

        /// <summary>
        /// Wait till we get out of the paused state.
        /// </summary>
        private void waitInPausedState()
        {
            if (m_paused)
            {
                string oldMessage = m_bInfo.m_Statistics.m_operationMsg;
                notifyOperation(CurrentOperation.Paused, "Pausing backup ");
                bool v = false;
                int reportPauseCountDown = 5;
                while (m_running && (v = m_pauseEvent.WaitOne(50)) == false)
                {
                    if (--reportPauseCountDown <= 0)
                    {
                        reportPauseCountDown = 5;
                        notifyOperation(CurrentOperation.InPause, "In Pause mode");
                    }
                }
                    
                if (m_running == false && v == false)
                {
                    // the while loop was terminated because of the running flag was set to false
                    // hence we need to reset the pause event
                    m_pauseEvent.Reset();
                    log.Debug("Pause event loop was exited without the pause_event being signalled, m_running=" + m_running + " pause_event signalled=" + v);
                }
                notifyOperation(CurrentOperation.Resumed, "Resuming backup");
                log.Debug("Resuming Backup thread");
                m_bInfo.m_Statistics.m_operationMsg = oldMessage;
            }
        }



        public void Stop( ) {
            bool v = m_running;
            // the force shutdown flag gets turned on if a user hits the stop button twice in a row
            // which indicates that the backup thread did not quit after a request for a graceful shutdown.
            bool forceShutdown = false;
            if (m_running == false)
            {
                forceShutdown = true;
            }
            m_running = false;
            m_paused = false;
            if (m_bInfo != null)
            {
                m_bInfo.m_Statistics.m_backupWasStopped = true;
            }
            m_pauseEvent.Set();
            if (m_backupThread!=null)
            {
                if(forceShutdown && m_backupThread.ThreadState == System.Threading.ThreadState.Running) {
                    m_backupThread.Abort();
                    m_backupThread = null;
                }
                else if (m_backupThread.ThreadState == System.Threading.ThreadState.Stopped ||
                    m_backupThread.ThreadState == System.Threading.ThreadState.Aborted)
                {
                    m_backupThread = null;
                }
            }
        }

        internal bool IsRunning( ) {
            bool v = (m_backupThread!=null && m_backupThread.IsAlive) ? true : false;
            return v; // running
        }

        /// <summary>
        /// Wait for currently running job to finish
        /// </summary>
        public void waitForJobToFinish()
        {
            while (IsRunning())
            {
                m_backupJobFinishEvent.WaitOne(5 * 1000);
            }
        }

        internal bool IsPaused()
        {
            return m_paused;
        }

        internal void Pause()
        {
            lock (this)
            {
                m_paused = true;
                m_bInfo.m_Job.registerRunPause();
            }
        }

        internal void Resume()
        {
            lock (this)
            {
                ResumeAndLog();
                m_pauseEvent.Set();
            }
        }

        private void ResumeAndLog()
        {
            lock (this)
            {
                m_paused = false;
                m_bInfo.m_Job.registerRunResume();
            }
        }

        /// <summary>
        /// Start the backup job after some initial checks
        /// </summary>
        /// <param name="items"></param>
        /// <param name="destDir"></param>
        /// <param name="job"></param>
        /// <param name="updateHandler"></param>
        /// <param name="notifyParam">The callback param when stats update handler is called</param>
        public void startBackupJob(BackupInfo binfo)
        {
            if(m_backupThread!=null && m_backupThread.IsAlive)
            {
                binfo.m_LastException = new BackupException("A backup job is already running");
                throw binfo.m_LastException;
            }
            m_bInfo = binfo;
            Stop();
            if (IsRunning())
            {
                binfo.m_LastException = new BackupException("Could not stop previous backup job");
                throw binfo.m_LastException;
            }
            m_backupThread = new Thread( new ParameterizedThreadStart(theRealBackupThread));
            m_backupThread.Name = m_backupThread.Name + "_Backup Thread job=<" + m_bInfo.m_Job.m_Id.m_Name + ">";
            m_bInfo.m_Statistics.resetCounters();
            m_backupThread.Priority = ThreadPriority.BelowNormal;
            m_backupThread.Start(binfo);
        }

        private void dirWalk2(string path, dirWalkCallback callback, ref bool keepGoing,int mode)
        {
            if (keepGoing)
            {
                FileSystemInfo thisEntry = null;
                FileSystemInfo[] entries = null;
                if(File.Exists(path)) {
                    FileInfo f = new FileInfo(path);
                    entries = new FileSystemInfo[] { f };
                } else if(Directory.Exists(path)) {
                    DirectoryInfo d = new DirectoryInfo(path);
                    entries = d.GetFileSystemInfos();
                    thisEntry = d;
                }

                bool skipRecursion = false;
                if (entries != null)
                {
                    foreach (FileSystemInfo e in entries)
                    {
                        bool isDir = (e.Attributes & FileAttributes.Directory) != 0;
                        if (mode == DIRWALK_TOP_LEVEL)
                        {
                            // allow the callback to check first if walking through the top level files/directories
                            if (!(keepGoing = callback(e, mode, ref skipRecursion)))
                            {
                                break;
                            }
                        }

                        if (isDir)
                        {
                            if (skipRecursion)
                            {
                                notifyOperation(CurrentOperation.StatusUpdate, "Skipping further recursion of directory <" + e.FullName + ">");
                            }
                            else
                            {
                                dirWalk2(e.FullName, callback, ref keepGoing, mode);
                            }
                        }
                        
                        if (mode != DIRWALK_TOP_LEVEL)
                        {
                            // call the observer after going into the dirwalk if not looking at top level entries
                            if(!(keepGoing = callback(e, mode, ref skipRecursion))) 
                            {
                                break;
                            }
                        }
                        skipRecursion = false;
                    }

                    if (thisEntry != null)
                    {
                        callback(thisEntry, mode, ref skipRecursion);
                    }
                }
            }
        }

        private void dirWalk(string path,dirWalkCallback callback, int mode)
        {
            bool keepGoing = true;
            dirWalk2(path, callback, ref keepGoing,mode);
        }
    }
}

